#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Time    : 2018/1/30 0030 9:49
# @Author  : Arliki
# @email   : hkdnxycz@outlook.com
# @File    : movie_url

import scrapy, time, re, demjson, redis
from scrapy_redis.spiders import RedisSpider
from qqcoment.items import MoviesItem

start = "http://list.video.qq.com/fcgi-bin/list_common_cgi?otype=json&novalue=1&platform=1&version=10000&intfname=web_vip_movie_new&tid=687&appkey=c8094537f5337021&appid=200010596&type=1&sourcetype=1&itype=-1&iyear=-1&iarea=-1&iawards=-1&sort=17&pagesize=30&offset=%s&callback=jQuery19107042628603705061_1517278039332&_=1517278039340"
offset = 30


class MovieSpider(RedisSpider):
    name = 'movie_url'
    redis_key = 'qqlive:movies'
    custom_settings = {
        'REDIS_PARAMS': {
            'db': 3,
            'password': '5tgbnhy67ujm'
        }
    }

    def __init__(self, *args, **kwargs):
        domain = kwargs.pop('domain', '')
        self.allowed_domains = filter(None, domain.split(','))
        super(MovieSpider, self).__init__(*args, **kwargs)

    def parse(self, response):
        global start, offset
        if response.status == 200:
            next_url = start % offset
            yield scrapy.Request(next_url, self.get_movies, dont_filter=True)

    def get_movies(self, response):
        global start, offset
        res = re.search(r'.*?\((.*?)\)$', response.body.decode('utf-8')).group(1)
        res = demjson.decode(res)
        movies = res['jsonvalue']['results']
        i = 0
        item = MoviesItem()
        for x in movies:
            i += 1
            url = "https://v.qq.com/x/cover/%s.html" % x['id']
            item['name'] = x['fields']['c_title']
            item['second_name'] = x['fields']['second_title']
            item['starts'] = x['fields']['opinion_score']
            item['url'] = url
            yield item
        if i < 30:
            pass
        else:
            nowset = re.search('.*offset=(\d+).*', response.url).group(1)
            offset = int(nowset) + 30
            next_url = start % offset
            yield scrapy.Request(next_url, self.get_movies, dont_filter=True)

            # lpush qqlive:movies http://www.arliki.com
            # https://v.qq.com/x/list/movie?sort=19&offset=0
